package com.chance.cc.crawler.development.scripts.pcauto.CarSeries;

import com.alibaba.fastjson.JSONObject;
import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.CrawlerRecordContext;
import com.chance.cc.crawler.core.downloader.HttpPage;
import com.chance.cc.crawler.core.pipeline.result.CrawlerData;
import com.chance.cc.crawler.core.record.CrawlerRecord;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.core.script.crawler.CrawlerCommonScript;
import com.chance.cc.crawler.core.selector.Json;
import com.chance.cc.crawler.core.selector.Selectable;
import com.chance.cc.crawler.core.tags.KVTag;
import com.chance.cc.crawler.core.tags.LabelTag;
import com.chance.cc.crawler.core.tags.crawler.CrawlerBusinessTags;
import com.chance.cc.crawler.development.scripts.allfeild.AutoVMCommonField;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;

import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerDataType.article;

public class PcautoCarSeriesCrawlerScript extends CrawlerCommonScript {
    private Logger log = LoggerFactory.getLogger(PcautoCarSeriesCrawlerScript.class);//日志
    private static final String DOMAIN = "pcauto";
    private static final String SITE = "carSeries";
    private static final String REQUEST_AGAIN_TAG = DOMAIN + "_request_again";

    //   https://car.autohome.com.cn/javascript/NewSpecCompare.js?20131010
    private static final String HomeRegulars = "https://car.autohome.com.cn/javascript/NewSpecCompare.js\\?20131010";//动态链接
    //    https://price.pcauto.com.cn/sg424/
    private static final String CarSeriesRegulars = "https://price.pcauto.com.cn/\\S*/";//车系链接
    //    https://bbs.pcauto.com.cn/forum-17025.html
    private static final String ForumRegulars = "https://bbs.pcauto.com.cn/forum-\\S*.html";//论坛链接
    //    https://price.pcauto.com.cn/sg424/all.html

    private static final String DiscontinuedRegulars = "https://price.pcauto.com.cn/\\S*/all.html";//停售链接
//    //    https://car.autohome.com.cn/config/spec/46839.html#pvareaid=3454495
//    private static final String ConfigRegulars = "https://car.autohome.com.cn/config/spec/\\d*.html";//配置链接

    @Override
    public String domain() {
        return DOMAIN;
    }

    @Override
    public void initUrlRegulars() {
        addUrlRegular(HomeRegulars);
        addUrlRegular(CarSeriesRegulars);
        addUrlRegular(ForumRegulars);
        addUrlRegular(DiscontinuedRegulars);
//        addUrlRegular(ConfigRegulars);
    }

    @Override
    public boolean crawlerCheck(CrawlerRequestRecord crawlerRecord) {
        String site = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("site");
        return SITE.equals(site);
    }

//    public List<CrawlerRecord> prepareRequest(CrawlerRequestRecord crawlerRequestRecord, List<CrawlerRequestRecord> supportSourceRecords) {
//        List<CrawlerRecord> prepareLinks = new ArrayList<>();
//        for (CrawlerRequestRecord supportSourceRecord : supportSourceRecords) {
//            String url = supportSourceRecord.getHttpRequest().getUrl();
//            if (url.contains("keys")) {
//                this.initKeyWord(crawlerRequestRecord, supportSourceRecord);
//            }
//        }
//        return prepareLinks;
//    }

    //取出代表不同的车系key
//    private void initKeyWord(CrawlerRequestRecord crawlerRequestRecord, CrawlerRequestRecord supportSourceRecord) {
//        HttpPage internalDownloadPage = supportSourceRecord.getInternalDownloadPage();
//        Json json = internalDownloadPage.getJson();
//        String msg = json.jsonPath($_type + ".msg").get();
//        if (!"success".equals(msg)) {
//            log.error("keyword page [{}] download error!", internalDownloadPage.getRequest().getUrl());
//            return;
//        }
//        List<String> keys = new ArrayList<>();
//        List<String> all = json.jsonPath($_type + ".content").all();
//        for (String data : all) {
//            JSONObject jsonObject = JSONObject.parseObject(data);
//            String keyword = jsonObject.getString("keyword");
//            keys.add(keyword);
//        }
//        crawlerRequestRecord.tagsCreator().bizTags().addCustomKV("keys", keys);
//    }

    @Override
    public List<CrawlerRequestRecord> parseLinks(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        List<CrawlerRequestRecord> parsedLinks = new ArrayList();
        //        判断页面是否加载成功
        String requestUrl = crawlerRecord.getHttpRequest().getUrl();
        if (page.getStatusCode() != 200 || (!page.isDownloadSuccess())) {
            log.error("download page url == {null} error status is {}", requestUrl, page.getStatusCode());
            if (page.getStatusCode() != 404) {//如果没有成功的下载  进行重新下载
                this.requestAgainCrawlerRecord(parsedLinks, crawlerRecord);
                crawlerRecord.setNeedWashPage(false);
                return parsedLinks;
            }
        }
//        if (crawlerRecord.getHttpRequest().getUrl().matches(HomeRegulars)) {
//            homeRequest(crawlerRecord, page, parsedLinks);//解析动态车系
//        } else
            if (crawlerRecord.getHttpRequest().getUrl().matches(CarSeriesRegulars)) {
            carSeriesRequest(crawlerRecord, page, parsedLinks);//解析详情和在售配置
        } else if (crawlerRecord.getHttpRequest().getUrl().matches(ForumRegulars)) {
            forumRequest(crawlerRecord, page, parsedLinks);//解析论坛
        }
//        else if (crawlerRecord.getHttpRequest().getUrl().matches(DiscontinuedRegulars) || crawlerRecord.getHttpRequest().getUrl().matches(DiscontinuedTwoRegulars)) {
//            configRequest(crawlerRecord, page, parsedLinks);//解析停售配置
//        }
        return parsedLinks;
    }

    //    解析动态车系
//    private void homeRequest(CrawlerRequestRecord crawlerRecord, HttpPage page, List<CrawlerRequestRecord> parsedLinks) {
//        KVTag keys = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTag("keys");
//        List<String> name = (List<String>) keys.getVal();
//        for (int i = 0; i < name.size(); i++) {//name.size()
//            String url = "https://price.pcauto.com.cn/sg424" + name.get(i)+"/";//得到车系页面
//            CrawlerRequestRecord record = CrawlerRequestRecord.builder()
//                    .itemPageRequest(crawlerRecord)
//                    .httpUrl(url)
//                    .releaseTime(Long.valueOf(System.currentTimeMillis()))
//                    .copyBizTags()
//                    .needWashed(false)
//                    .build();
//            parsedLinks.add(record);
//        }
//    }

    private void carSeriesRequest(CrawlerRequestRecord crawlerRecord, HttpPage page, List<CrawlerRequestRecord> parsedLinks) {
        String carSeriesUrl = page.getRequest().getUrl();//车系url
        String forumUrl = null;
        String forumStr = page.getHtml().xpath("//ul[@class=\"channel\"]/li[12]/a/@href").get();
        if (forumStr!=null){
            forumUrl = "https:" + forumStr;  //论坛url
        }
        String koubeiUrl = "https:" + page.getHtml().xpath("//ul[@class=\"channel\"]/li[7]/a/@href").get();//口碑url
        String brandName = page.getHtml().xpath("//div[@class=\"pos-mark\"]/a[3]/text()").get();//品牌名
        String brandUrl = "https://price.pcauto.com.cn" + page.getHtml().xpath("//div[@class=\"pos-mark\"]/a[3]/@href").get();//品牌url
        String subsidiaryBrandName = page.getHtml().xpath("//div[@class=\"pos-mark\"]/a[4]/text()").get();//子品牌名
        String carSeriesName = page.getHtml().xpath("//div[@class=\"pos-mark\"]/a[5]/text()").get();//车系名
        String carSeriesName2 = null;
        if (carSeriesName != null) {
            carSeriesName2 = subsidiaryBrandName +"-"+ carSeriesName;//车系2
        }
        crawlerRecord.tagsCreator().bizTags().addCustomKV(AutoVMCommonField.Field_Series_url, carSeriesUrl);//车系url
        HashMap<String, String> seriesMap = new HashMap<>();
        seriesMap.put(AutoVMCommonField.Field_Series_name, carSeriesName);//车系名
        seriesMap.put(AutoVMCommonField.Field_Series_url, carSeriesUrl);//车系url
        crawlerRecord.tagsCreator().bizTags().addCustomKV(AutoVMCommonField.Tag_Field_Series, seriesMap);//车系
        HashMap<String, String> brandMap = new HashMap<>();
        brandMap.put(AutoVMCommonField.Field_Brand_name, brandName);//品牌名
        brandMap.put(AutoVMCommonField.Field_Brand_url, brandUrl);//品牌url
        crawlerRecord.tagsCreator().bizTags().addCustomKV(AutoVMCommonField.Tag_Field_Brand, brandMap);//品牌
        crawlerRecord.tagsCreator().bizTags().addCustomKV(AutoVMCommonField.Field_Praise_url, koubeiUrl);//口碑
        crawlerRecord.tagsCreator().bizTags().addCustomKV(AutoVMCommonField.Field_Forum_url, forumUrl);//论坛url
        crawlerRecord.tagsCreator().bizTags().addCustomKV(AutoVMCommonField.Tag_Field_Series_Two, carSeriesName2);//二级车系
        crawlerRecord.tagsCreator().bizTags().addCustomKV("subsidiaryBrandName", subsidiaryBrandName);//子品牌

        if (forumUrl!=null){
            CrawlerRequestRecord forumRecord = CrawlerRequestRecord.builder()//解析论坛
                    .itemPageRequest(crawlerRecord)
                    .recordKey(forumUrl)
                    .httpUrl(forumUrl)
                    .releaseTime(System.currentTimeMillis())
                    .copyBizTags()
                    .build();
            parsedLinks.add(forumRecord);
        }

//        List<String> configUrls = page.getHtml().xpath("//a[./text()=\"配置\"]/@href").all();//所有在售配置
//        for (String configUrlStr : configUrls) {
//            String configId = Pattern.compile("[^0-9]").matcher(configUrlStr).replaceAll(" ").replaceAll("\\s+", " ").trim().split(" ")[0];//url的关键字
//            //https://car.autohome.com.cn/config/spec/46839.html#pvareaid=3454495
//            String configUrl = "https://car.autohome.com.cn/config/spec/" + configId + ".html";
//            CrawlerRequestRecord configurationRecord = CrawlerRequestRecord.builder()//解析在售
//                    .itemPageRequest(crawlerRecord)
//                    .recordKey(configUrl)
//                    .httpUrl(configUrl)
//                    .releaseTime(System.currentTimeMillis())
//                    .resultLabelTag(article)
//                    .needWashed(true)
//                    .copyBizTags()
//                    .build();
//            parsedLinks.add(configurationRecord);
//        }
    }


    private void forumRequest(CrawlerRequestRecord crawlerRecord, HttpPage page, List<CrawlerRequestRecord> parsedLinks) {
        String forumUrl = page.getRequest().getUrl();
        String forumName = page.getHtml().xpath("//div[@class=\"com-crumb\"]/a[4]").get();//论坛名
        HashMap<String, String> forumMap = new HashMap<>();
        forumMap.put(AutoVMCommonField.Field_Forum_name, forumName);
        forumMap.put(AutoVMCommonField.Field_Forum_url, forumUrl);
        crawlerRecord.tagsCreator().bizTags().addCustomKV(AutoVMCommonField.Tag_Forum_detail, forumMap);//论坛

//        https://price.pcauto.com.cn/sg424/all.html
        String carSeriesUrl = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal(AutoVMCommonField.Field_Series_url);//车型
        String[] articleId = carSeriesUrl.split("/");
        String saleUrl="https://price.pcauto.com.cn/"+articleId[articleId.length-1]+"/all.html";
        CrawlerRequestRecord saleRecord = CrawlerRequestRecord.builder()//解析车型
                    .itemPageRequest(crawlerRecord)
                    .recordKey(saleUrl)
                    .httpUrl(saleUrl)
                    .releaseTime(System.currentTimeMillis())
                    .resultLabelTag(article)
                    .copyBizTags()
                    .needWashed(true)
//                    .notFilterRecord()
                    .build();
            saleRecord.tagsCreator().bizTags().addCustomKV(AutoVMCommonField.Tag_Forum_detail, forumMap);
            parsedLinks.add(saleRecord);
        }


//    //    解析配置
//    private void configRequest(CrawlerRequestRecord crawlerRecord, HttpPage page, List<CrawlerRequestRecord> parsedLinks) {
//        String url = page.getRequest().getUrl();//获取url
//
//        List<Selectable> nodes = page.getHtml().xpath("//td[@class=\"info_d\"]").nodes();//所有停售款
//        for (Selectable node : nodes) {
//            String configUrlStr = node.xpath(".//a[./text()=\"参数配置\"]/@href").get();//停售配置
//            String configId = Pattern.compile("[^0-9]").matcher(configUrlStr).replaceAll(" ").replaceAll("\\s+", " ").trim().split(" ")[0];//url的关键字
//            //https://car.autohome.com.cn/config/spec/46839.html#pvareaid=3454495
//            String configUrl = "https://car.autohome.com.cn/config/spec/" + configId + ".html";
//            CrawlerRequestRecord configRecord = CrawlerRequestRecord.builder()//解析停售
//                    .itemPageRequest(crawlerRecord)
//                    .recordKey(configUrl)
//                    .httpUrl(configUrl)
//                    .releaseTime(System.currentTimeMillis())
//                    .resultLabelTag(article)
//                    .needWashed(true)
//                    .copyBizTags()
//                    .build();
//            parsedLinks.add(configRecord);
//        }
//    }


    @Override
    public List<CrawlerData> washPage(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        List<CrawlerData> crawlerDataList = new ArrayList<>();
        if (crawlerRecord.getHttpRequest().getUrl().matches(DiscontinuedRegulars)) {
            if (crawlerRecord.tagsCreator().resultTags().hasDataType(article)) {
                crawlerDataList.addAll(saleWash(crawlerRecord, page));
            }
        }
//        if (crawlerRecord.getHttpRequest().getUrl().matches(DiscontinuedRegulars) || crawlerRecord.getHttpRequest().getUrl().matches(DiscontinuedTwoRegulars)) {
//            if (crawlerRecord.tagsCreator().resultTags().hasDataType(article)) {
//                crawlerDataList.addAll(discontinuedWash(crawlerRecord, page));
//            }
//        }
//        if (crawlerRecord.getHttpRequest().getUrl().matches(ConfigRegulars)) {
//            if (crawlerRecord.tagsCreator().resultTags().hasDataType(article)) {
//                crawlerDataList.addAll(configWash(crawlerRecord, page));
//            }
//        }
        return crawlerDataList;
    }

    //    解析在售款
    public List<CrawlerData> saleWash(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        List<CrawlerData> crawlerArticleDataList = new ArrayList<>();
        String carSeriesUrl = page.getRequest().getUrl();//车系url
//            https://price.pcauto.com.cn/m108244/
        List<String> saleUrls = page.getHtml().xpath("//table//a[@class=\"blue carInfo_tdA\"]/@href").all();//车型url
        List<String> saleTitles = page.getHtml().xpath("//table//a[@class=\"blue carInfo_tdA\"]/text()").all();//在售标题名
        List<String> sales = page.getHtml().xpath("//table//td[@class=\"t_r\"]/text()[1]").all();//状态
        for (int i = 0; i < saleUrls.size(); i++) {
            String saleUrl = "https://price.pcauto.com.cn" + saleUrls.get(i);
            String saleTitle = saleTitles.get(i);
            String saleStr = sales.get(i);
            int sale=1;
            if (saleStr!=null){
                if(saleStr.contains("停售")){
                    sale=0;
                }
            }
            //https://price.pcauto.com.cn/sg424/y42812.html
            String[] articleId = carSeriesUrl.split("/");
            CrawlerData crawlerData = CrawlerData.builder()
                    .data(crawlerRecord, page)
                    .dataId(StringUtils.joinWith("-", domain(), article, articleId[articleId.length-2]))
                    .resultLabelTag(CrawlerEnum.CrawlerDataType.valueOf("article"))
                    .url(carSeriesUrl)
                    .addContentKV(AutoVMCommonField.Field_Model_name, saleTitle)
                    .addContentKV(AutoVMCommonField.Field_Model_url, saleUrl)
                    .addContentKV(AutoVMCommonField.Tag_Is_sale, "sale")
                    .flowInPipelineTag("kafka")
                    .build();
            crawlerData.setFilterPipelineResult(true);
            crawlerArticleDataList.add(crawlerData);
        }

        return crawlerArticleDataList;
    }


    //    冲洗配置款
//    public List<CrawlerData> configWash(CrawlerRequestRecord crawlerRecord, HttpPage page) {
//        List<CrawlerData> crawlerArticleDataList = new ArrayList<>();
//        List<Map<String, Map<String, String>>> configList = new ArrayList<>();
//        Map<String, Map<String, String>> map = new HashMap<>();
//        Map<String, String> configMap = new HashMap<>();
//        String configUrl = page.getRequest().getUrl();
//        String configId = Pattern.compile("[^0-9]").matcher(configUrl).replaceAll("").trim();//url的关键字
//        String configJson = "";
//        String optionJson = "";
//        String colorJson = "";
//        String innerColorJson = "";
//        try {
//            String config = page.getRawText().split("var config =")[1].split("var option")[0].trim();
//            configJson = config.substring(0, config.length() - 1);//70
//            String option = page.getRawText().split("var option =")[1].split("var color")[0].trim();
//            optionJson = option.substring(0, option.length() - 1);//159
//            String color = page.getRawText().split("var color =")[1].split("var innerColor")[0].trim();
//            colorJson = color.substring(0, color.length() - 1);//1
//            String innerColor = page.getRawText().split("var innerColor =")[1].split("var bag")[0].trim();
//            innerColorJson = innerColor.substring(0, innerColor.length() - 1);//1
//        } catch (Exception e) {
//            log.error(e.getMessage(), e + "该网页可能不存在");
//        }
//        if (colorJson != null && !"".equals(colorJson)) {
//            List<Selectable> configNodes = new Json(configJson).jsonPath($_type + "..paramtypeitems[*]").nodes();
//            for (Selectable type : configNodes) {
//                String types = new Json(type.get()).jsonPath(".name").get();//分类
//                List<Selectable> configNode = new Json(type.get()).jsonPath(".paramitems[*]").nodes();
//                configMap = new HashMap<String, String>();
//                for (Selectable node : configNode) {
//                    String configName = new Json(node.get()).jsonPath(".name").get();//k值
//                    String configVal = new Json(node.get()).jsonPath(".valueitems[0].value").get();//v值
//                    configMap.put(configName, configVal);
//                }
//                map.put(types, configMap);
//            }
//
//            List<Selectable> optionNodes = new Json(optionJson).jsonPath($_type + "..configtypeitems[*]").nodes();
//            for (Selectable type : optionNodes) {
//                String types = new Json(type.get()).jsonPath(".name").get();//分类
//                List<Selectable> optionNode = new Json(type.get()).jsonPath(".configitems[*]").nodes();
//                configMap = new HashMap<String, String>();
//                for (Selectable node : optionNode) {
//                    String configName = new Json(node.get()).jsonPath(".name").get();//k值
//                    String configVal = new Json(node.get()).jsonPath(".valueitems[0].value").get();//v值
//                    configMap.put(configName, configVal);
//                }
//                map.put(types, configMap);
//            }
//
//            configMap = new HashMap<String, String>();
//            List<Selectable> colorStr = new Json(colorJson).jsonPath($_type + "..specitems[*]").nodes();
//            for (Selectable type : colorStr) {
//                String id = new Json(type.get()).jsonPath(".specid").get();//分类
//                if (configId == id) {
//                    List<Selectable> colorNode = new Json(type.get()).jsonPath(".coloritems[*]").nodes();
//                    StringBuffer configName = new StringBuffer();//将文本拼接
//                    for (Selectable node : colorNode) {
//                        List<String> name = new Json(node.get()).jsonPath(".name").all();//k值
////                    String configVal = new Json(node.get()).jsonPath(".value").get();//v值
//                        for (String text : name) {
//                            configName.append(text).append(" ");
//                        }
//                    }
//                    configMap.put("外观颜色", configName.toString());
//                }
//            }
//
//            List<Selectable> innerColorStr = new Json(innerColorJson).jsonPath($_type + "..specitems[*]").nodes();
//            for (Selectable type : innerColorStr) {
//                String id = new Json(type.get()).jsonPath(".specid").get();//分类
//                if (configId == id) {
//                    List<Selectable> colorNode = new Json(type.get()).jsonPath(".coloritems[*]").nodes();
//                    StringBuffer configName = new StringBuffer();//将文本拼接
//                    for (Selectable node : colorNode) {
//                        List<String> name = new Json(node.get()).jsonPath(".name").all();//k值
////                    String configVal = new Json(node.get()).jsonPath(".value").get();//v值
//                        for (String text : name) {
//                            configName.append(text).append(" ");
//                        }
//                    }
//                    configMap.put("内饰颜色", configName.toString());
//                }
//            }
//            map.put("选装包", configMap);
//            configList.add(map);
//
//            String a = JSONObject.toJSON(configList).toString();
//            String config = a.replaceAll("<span class='", "").replaceAll("'></span>", " ").trim();
//
//            CrawlerData crawlerData = CrawlerData.builder()
//                    .data(crawlerRecord, page)
//                    .dataId(StringUtils.joinWith("-", domain(), article, configId))
//                    .resultLabelTag(CrawlerEnum.CrawlerDataType.valueOf("article"))
//                    .url(configUrl)
//                    .flowInPipelineTag("kafka")
//                    .addContentKV("config", config)
//                    .build();
//            crawlerData.setFilterPipelineResult(true);
//            crawlerArticleDataList.add(crawlerData);
//
//        }
//        return crawlerArticleDataList;
//    }

    @Override
    public void afterExecute(CrawlerRecordContext context) {
    }

    /*
     * 重新下载
     * */
    private void requestAgainCrawlerRecord(List<CrawlerRequestRecord> crawlerRequestRecords, CrawlerRequestRecord crawlerRecord) {
        CrawlerBusinessTags crawlerBusinessTags = crawlerRecord.tagsCreator().bizTags();
        int count = 0;
        if (crawlerBusinessTags.hasKVTag(REQUEST_AGAIN_TAG)) {
            count = (int) crawlerBusinessTags.getCategoryTag().getKVTag(REQUEST_AGAIN_TAG).getVal();
            if (count >= 10) {
                log.error("pcauto download page the number of retries exceeds the limit" +
                        ",request url {}", crawlerRecord.getHttpRequest().getUrl());
                return;
            }
        }

        String requestUrl = crawlerRecord.getHttpRequest().getUrl();
        CrawlerRequestRecord crawlerRequestRecord = null;
        LabelTag type = crawlerRecord.tagsCreator().requestTags().getCategoryTag().getLabelTags().get("turn_page_item_request");
        if (type == null) {
            crawlerRequestRecord = CrawlerRequestRecord.builder()
                    .turnPageRequest(crawlerRecord)
                    .httpUrl(requestUrl)
                    .recordKey(crawlerRecord.getRecordKey() + count)
                    .releaseTime(crawlerRecord.getReleaseTime())
                    .notFilterRecord()
                    .copyBizTags()
                    .copyResultTags()
                    .build();
        } else {
            crawlerRequestRecord = CrawlerRequestRecord.builder()
                    .itemPageRequest(crawlerRecord)
                    .httpUrl(requestUrl)
                    .recordKey(crawlerRecord.getRecordKey() + count)
                    .releaseTime(crawlerRecord.getReleaseTime())
                    .copyBizTags()
                    .copyResultTags()
                    .build();
        }

        crawlerRequestRecord.setNeedWashPage(crawlerRecord.isNeedWashPage());
        crawlerRequestRecord.setNeedParsedPage(crawlerRecord.isNeedParsedPage());
        crawlerRequestRecord.getHttpRequest().setHeaders(crawlerRecord.getHttpRequest().getHeaders());
        crawlerRequestRecord.getHttpRequest().setExtras(crawlerRecord.getHttpRequest().getExtras());
        crawlerRequestRecord.tagsCreator().bizTags().addCustomKV(REQUEST_AGAIN_TAG, ++count);
        crawlerRequestRecords.add(crawlerRequestRecord);
    }
}
